package fetcher

import (
	"net/http"
	"fmt"
	"golang.org/x/text/transform"
	"io/ioutil"
	"io"
	"golang.org/x/text/encoding"
	"bufio"
	"golang.org/x/net/html/charset"
	"golang.org/x/text/encoding/unicode"
	"log"
)

func fetcher (url string) ([]byte, error) {
	resp, err := http.Get(url)
	
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	
	if resp.StatusCode != http.StatusOK {
		return nil,
			fmt.Errorf("error: status code:", resp.StatusCode)
	}
	e := determineEncoding(resp.Body)
	utf8Reader := transform.NewReader(resp.Body, e.NewDecoder())
	//for r, val := range resp.Header {
	//	fmt.Println(r, val)
	//	fmt.Println()
	//}
	//fmt.Println(resp.Header)
	
	return ioutil.ReadAll(utf8Reader)
}


func determineEncoding(r io.Reader) encoding.Encoding {
	bytes, err := bufio.NewReader(r).Peek(1024)
	
	if err != nil {
		log.Printf("Fetcher error: %v", err)
		return unicode.UTF8
	}
	
	e, _, _ := charset.DetermineEncoding(bytes, "")
	return e
}
